/*
* This file is a part of the open-eBackup project.
* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
* If a copy of the MPL was not distributed with this file, You can obtain one at
* http://mozilla.org/MPL/2.0/.
*
* Copyright (c) [2024] Huawei Technologies Co.,Ltd.
*
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
*/
#include <set>
#include <map>
#include <sys/socket.h>
#include <netdb.h>
#include <cstdio>
#include <arpa/inet.h>
#include <cerrno>
#include <cstring>
#include <atomic>
#include <common/JsonHelper.h>
#include "common/Structs.h"
#include "common/uuid/Uuid.h"
#include "common/utils/Utils.h"
#include "protect_engines/hcs/common/HcsHttpStatus.h"
#include "volume_handlers/oceanstor/OceanStorVolumeHandler.h"
#include "volume_handlers/fusionstorage/FusionStorageVolumeHandler.h"
#include "volume_handlers/fusionstorage/FusionStorageIscsiDiskScanner.h"
#include "job_controller/jobs/backup/BackupJob.h"
#include "protect_engines/hcs/utils/HcsOpServiceUtils.h"
#include "HCSProtectEngine.h"
#include "protect_engines/openstack/api/cinder/model/DeleteSnapshotResponse.h"
#include "protect_engines/openstack/api/cinder/model/DeleteSnapshotRequest.h"

using namespace OpenStackPlugin;

namespace {
const std::string MODULE = "HCSProtectEngine";
const std::string INTERNAL_BACKUP_SCENCE = "1";
const int32_t WAIT_TIMES = 30;
const int32_t WAIT_INTERVAL = 10;
constexpr uint32_t GB_SIZE = 1024 * 1024 * 1024UL;
const std::string FUSIONSTORAGE_TYPE = "FusionStorage";  // FusionStorage、OceanPacific
const std::string OCEANSTORE_TYPE = "OceanStor";         // OceanStorV3、OceanStorV5
const std::string DORADO_TYPE = "Dorado";                // DoradoV6
const std::string RESOURCE_TYPE_KEY = "sourceType";
const std::string RESOURCE_TYPE_TENANT = "tenant";
const std::string RESOURCE_TYPE_PROJECT = "project";
const std::string RESOURCE_TYPE_VM = "vm";
const std::string RESOURCE_TYPE_DISK = "disk";
const std::string RESOURCE_TYPE_VDC = "vdc";
/* cert type */
const std::string CPS_CERT = "CPS Cert(cinder.pem)";
const std::string CINDER_CERT_HEAD = "cinder-";
/* volume type definition */
const std::string HCS_VOLTYPE_SHARED = "shared";
const int32_t BUF_LIMT = 1024;
const int32_t CHECK_HCS_CONNECT_FAILED = 1577210003;
const int32_t CHECK_STORAGE_CONNECT_FAILED = 1577213520;  // 检查和存储连通性失败
const int32_t CHECK_VBS_CONFIG_FAILED = 1577213543; // 检查VBS配置文件失败
const int32_t HCS_CERT_NOT_EXIST = 1577090049;  // HCS证书检验不通过，（{0}）证书不存在
const std::string VDC_QUERY_RESOURCE_FAILED = "1577210099"; // VDC列表中的VDC管理员（{0}）帐号信息异常。
const int32_t CHECK_HCS_TENANT_FAILED = 1577210003; // 通过注册的管理员触发重新扫描，返回错误信息
const int32_t QUERY_PROJECTS_LIST_FAILED = 1577209857; // 通过最高级VDC管理员获取资源集时失败
const int32_t SCAN_CLOUD_SEVER_FAILED = 1577209873; // 通过项目扫描云服务器/磁盘失败
static std::atomic<std::int32_t> g_curCreateSnapshotValue { 0 };  // 单代理创建快照限制
static std::mutex m_mutex;
const int32_t CREATE_SNAPSHOT_WAIT_TIME = 10;  // unit: s
const int32_t CREATE_SNAPSHOT_RETRY_WAIT_TIME = 20;  // unit: s
const std::string STORAGE_TYPE_CENTRALIZED = "0";
const std::string STORAGE_TYPE_DISTRIBUTED = "1";
const int32_t MIN_PACIFIC_VERSION_INCREASE = 813;  // 支持使用rest接口创建差量卷的最小pacific版本
const int STANDARD_START_TIME_1970 = 1970;
const int CONSTANT_NUM_6 = 6;
const std::string CHECK_STORAGE_USAGE_FAILED_LABEL = "virtual_plugin_cnware_storage_usage_limit_label";
const double PERCENTAGE_ONE = 0.01;
const int32_t NO_STORAGE_INFO = 1577213586;
const int32_t VM_NOT_EXSIT = 1577213594;
const int32_t MAX_EXEC_COUNT = 5;
const int32_t COMMON_WAIT_TIME = 10;
}  // namespace


namespace HcsPlugin {

bool HCSProtectEngine::FormHttpRequest(ModelBase &request, bool useAdmin)
{
    m_hcsOpServiceUtils.GetAppEnv(m_appEnv);
    if (m_hcsOpServiceUtils.GetIsOpServiceEnv()) {
        INFOLOG("The environment is opservice");
        return FormHttpRequestInOpService(request, m_appEnv);
    }

    return FormHttpRequestInCommonScene(request, useAdmin);
}

bool HCSProtectEngine::FormHttpRequestInCommonScene(ModelBase &request, bool useAdmin)
{
    ProtectEnvExtendInfo backExtendInfo;
    if (!Module::JsonHelper::JsonStringToStruct(m_appEnv.extendInfo, backExtendInfo)) {
        ERRLOG("Get backup job extend info failed, %s", m_taskId.c_str());
        return false;
    }

    ProtectEnvAuthExtendInfo envAuthExtend;
    if (!Module::JsonHelper::JsonStringToStruct(m_appEnv.auth.extendInfo, envAuthExtend)) {
        ERRLOG("Get vdc info failed, %s", m_taskId.c_str());
        return false;
    }

    if (!Module::JsonHelper::JsonStringToStruct(envAuthExtend.m_vdcInfoStr, envAuthExtend.m_vdcInfo)) {
        ERRLOG("Get vdc info failed, %s", m_taskId.c_str());
        return false;
    }
    INFOLOG("Set request param, region: %s, scope(projectId): %s, endpoint: %s, vdc: %s. taskId: %s",
        backExtendInfo.m_regionId.c_str(), backExtendInfo.m_projectId.c_str(), m_appEnv.endpoint.c_str(),
        envAuthExtend.m_vdcInfo.m_name.c_str(), m_taskId.c_str());
    request.SetRegion(backExtendInfo.m_regionId);
    request.SetEndpoint(m_appEnv.endpoint);
    AuthObj cinderAuth;
    AuthObj iamAuth;
    if (!ParseCinderCert(m_appEnv.auth.extendInfo)) {
        ERRLOG("Failed to parse cinder cert.");
        return false;
    }

    if (useAdmin) {
        cinderAuth = { m_appEnv.auth.authkey, m_appEnv.auth.authPwd,
            m_cinderCertMgr->GetCertPath(), m_cinderCertMgr->GetRevocationListPath(), m_cinderCertMgr->IsVerifyCert() };
        iamAuth.name = m_appEnv.auth.authkey;
        iamAuth.passwd = m_appEnv.auth.authPwd;
        request.SetScopeValue("mo_bss_admin");
        request.SetDomain("mo_bss_admin");
    } else {
        cinderAuth = { envAuthExtend.m_vdcInfo.m_name, envAuthExtend.m_vdcInfo.m_passwd,
            m_cinderCertMgr->GetCertPath(), m_cinderCertMgr->GetRevocationListPath(), m_cinderCertMgr->IsVerifyCert() };
        iamAuth.name = envAuthExtend.m_vdcInfo.m_name;
        iamAuth.passwd = envAuthExtend.m_vdcInfo.m_passwd;
        request.SetScopeValue(backExtendInfo.m_projectId);
        request.SetDomain(envAuthExtend.m_vdcInfo.m_domainName);
    }
    request.SetUserInfo(cinderAuth);
    request.SetIamUserInfo(iamAuth);
    request.SetEnvAddress(m_appEnv.id);
    return true;
}

// op服务化场景组装请求，无需鉴权信息，token从扩展信息中读取
bool HCSProtectEngine::FormHttpRequestInOpService(ModelBase &request, ApplicationEnvironment &appEnv)
{
    request.SetRegion(m_hcsOpServiceUtils.GetRegion());
    request.SetEndpoint(m_hcsOpServiceUtils.GetDomain());
    request.SetEnvAddress(m_appEnv.id);
    request.SetScopeValue(m_hcsOpServiceUtils.GetProjectId());
    request.SetToken(m_hcsOpServiceUtils.GetToken());
    request.SetDomain("mo_bss_admin");
    request.SetExtendInfo(m_application.extendInfo);
    return true;
}

int HCSProtectEngine::PreHook(const ExecHookParam &para)

{
    if (!InitJobPara()) {
        ERRLOG("Init Job Para failed.");
        return FAILED;
    }
    if (m_jobHandle->GetJobType() != JobType::RESTORE) {
        INFOLOG("Job type is not RESTORE, no need to do prehook.");
        return SUCCESS;
    }
    if (para.stage == JobStage::EXECUTE_SUB_JOB) {
        m_reportParam = {
            "virtual_plugin_restore_job_execute_subjob_start_label",
            JobLogLevel::TASK_LOG_INFO,
            SubJobStatus::RUNNING, 0, 0 };
    }
    if (para.stage == JobStage::EXECUTE_POST_SUB_JOB || para.stage == JobStage::POST_JOB) {
        UnLockMachine();
    }
    return SUCCESS;
}

int HCSProtectEngine::LockMachine()
{
    LockServerRequest request;
    request.SetServerId(m_jobHandle->GetApp().id);
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    std::shared_ptr<LockServerResponse> response = m_ecsClient.LockServer(request);
    if (response == nullptr) {
        ERRLOG("Lock response is nullptr, %s", m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::ACCEPTED)) {
        ERRLOG("Lock server failed, status code: %d, %s", response->GetStatusCode(), m_taskId.c_str());
        return FAILED;
    }
    INFOLOG("Lock machine success.%s, %s", m_jobHandle->GetApp().id.c_str(), m_taskId.c_str());
    return SUCCESS;
}

int HCSProtectEngine::UnLockMachine()
{
    UnLockServerRequest request;
    request.SetServerId(m_jobHandle->GetApp().id);
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    std::shared_ptr<UnLockServerResponse> response = m_ecsClient.UnLockServer(request);
    if (response == nullptr) {
        ERRLOG("UnLock response is nullptr, %s", m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::ACCEPTED)) {
        ERRLOG("UnLock server failed, status code: %u, %s", response->GetStatusCode(), m_taskId.c_str());
        return FAILED;
    }
    INFOLOG("Unlock machine success,%s", m_taskId.c_str());
    return SUCCESS;
}

/**
 * @brief 前置PostHook锁定虚拟机
 *
 * @param para
 * @return int
 */
int HCSProtectEngine::PostHook(const ExecHookParam &para)
{
    if (m_jobHandle->GetJobType() == JobType::RESTORE && para.stage == JobStage::PRE_PREREQUISITE) {
        LockMachine();
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::QuerySnapshotExists(SnapshotInfo &snapshot)
{
    if (snapshot.m_volSnapList.empty()) {
        ERRLOG("No snapshot provided to query, %s", m_taskId.c_str());
        return FAILED;
    }

    snapshot.m_deleted = true;

    for (auto &volSnap : snapshot.m_volSnapList) {
        if (!ShowVolumeSnapshot(volSnap)) {
            ERRLOG("Show volume snapshot failed. Volume id: %s, snapshot id: %s. %s", volSnap.m_volUuid.c_str(),
                volSnap.m_snapshotId.c_str(), m_taskId.c_str());
            return FAILED;
        }
        if (!volSnap.m_deleted) {
            snapshot.m_deleted = false;
        }
    }

    return SUCCESS;
}

int32_t HCSProtectEngine::GetVolumeHandler(const VolInfo &volInfo, std::shared_ptr<VolumeHandler> &volHandler)
{
    if (!InitJobPara()) {
        ERRLOG("Initialize job para failed, %s", m_taskId.c_str());
        return FAILED;
    }
    m_hcsOpServiceUtils.GetAppEnv(m_appEnv);
    DBGLOG("Volume located storage type: %s. %s", volInfo.m_datastore.m_type.c_str(), m_taskId.c_str());
    if (volInfo.m_datastore.m_type.find(FUSIONSTORAGE_TYPE) != std::string::npos) {
        DBGLOG("Volume storage ip: %s, poolId: %s. %s", volInfo.m_datastore.m_ip.c_str(),
            volInfo.m_datastore.m_poolId.c_str(), m_taskId.c_str());
        std::shared_ptr<VirtPlugin::FusionStorageVolumeHandler> storageVolHandler =
            std::make_shared<VirtPlugin::FusionStorageVolumeHandler>(GetJobHandle(), volInfo, m_jobId, m_subJobId);
        storageVolHandler->SetOpService(m_hcsOpServiceUtils.GetIsOpServiceEnv());
        int32_t ret = storageVolHandler->InitializeVolumeInfo();
        if (ret != SUCCESS) {
            ERRLOG("Initialize volume info failed, %s", m_taskId.c_str());
            return ret;
        }
        volHandler = storageVolHandler;
        return SUCCESS;
    } else if (volInfo.m_datastore.m_type.find(OCEANSTORE_TYPE) != std::string::npos ||
        volInfo.m_datastore.m_type.find(DORADO_TYPE) != std::string::npos) {
        std::shared_ptr<VirtPlugin::OceanStorVolumeHandler> storageVolHandler =
            std::make_shared<VirtPlugin::OceanStorVolumeHandler>(GetJobHandle(), volInfo, m_jobId, m_subJobId);
        storageVolHandler->SetOpService(m_hcsOpServiceUtils.GetIsOpServiceEnv());
        if (!m_noNeedInitOceanVolHandler && storageVolHandler->InitializeVolumeInfo() != SUCCESS) {
            ERRLOG("Initialize volume info failed, %s", m_taskId.c_str());
            return FAILED;
        }
        volHandler = storageVolHandler;
    } else {
        ERRLOG("Get volume handler failed, %s", m_taskId.c_str());
        return FAILED;
    }

    DBGLOG("Get volume handler success, %s", m_taskId.c_str());
    return SUCCESS;
}


bool HCSProtectEngine::GetVolDetails(const VolInfo &volInfo, HSCVolDetail &volDetail)
{
    ShowVolumeRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }
    request.SetVolumeId(volInfo.m_uuid);
    std::shared_ptr<ShowVolumeResponse> response = m_evsClient.ShowVolumeDetail(request);
    if (response == nullptr || !response->Success()) {
        ERRLOG("Send get volume details request failed, volId: %s, %s", volInfo.m_uuid.c_str(), m_taskId.c_str());
        return false;
    }
    volDetail = response->GetHSCVolDetail();
    return true;
}

int32_t HCSProtectEngine::DetachVolume(const VolInfo &volObj)
{
    int ret = DetachVolumeHandle(volObj);
    TP_START("TP_HCS_DetachVolumeFailed", 1, &ret);
    TP_END
    if (ret != SUCCESS) {
        m_reportArgs = { volObj.m_uuid.c_str() };
        m_reportParam = {
            "virtual_plugin_restore_job_detach_volume_failed_label",
            JobLogLevel::TASK_LOG_ERROR,
            SubJobStatus::FAILED, 0, 0 };
        return ret;
    }
    m_reportArgs = { m_jobHandle->GetApp().id };
    m_reportParam = {
        "virtual_plugin_restore_job_detach_volume_success_label",
        JobLogLevel::TASK_LOG_INFO,
        SubJobStatus::RUNNING, 0, 0 };
    return SUCCESS;
}

int32_t HCSProtectEngine::DetachVolumeHandle(const VolInfo &volObj)
{
    HSCVolDetail volDetail;
    if (!GetVolDetails(volObj, volDetail)) {
        ERRLOG("Get volume details failed, %s", m_taskId.c_str());
        return FAILED;
    }
    // 判断卷是否已卸载
    if (volDetail.m_hostVolume.m_status == VOLUME_STATUS_AVAILABLE) {
        INFOLOG("Volume has been detached, volId: %s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return SUCCESS;
    }
    // 判断卷是否挂载与指定的虚拟机上
    bool findVolInServer = false;
    for (auto &attachPoint : volDetail.m_hostVolume.m_attachPoints) {
        if (m_jobHandle->GetApp().id == attachPoint.m_serverId) {
            findVolInServer = true;
            INFOLOG("Find volume in server, volId: %s, serverId: %s", volObj.m_uuid.c_str(), attachPoint.m_serverId.c_str());
            break;
        }
    }
    if (!findVolInServer) {
        ERRLOG("Volume does not attach to the server, volId: %s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    if (CheckDetachServerVolume(volObj) != SUCCESS) {
        return FAILED;
    }
    // 查询卷的状态是否为已挂载
    if (CheckVolumeStatus(volObj, VOLUME_STATUS_AVAILABLE) != SUCCESS) {
        ERRLOG("Detach server volume failed after query status, volId=%s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    DBGLOG("Volume(%s) detach success, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckVolumeStatus(const VolInfo &volObj, const std::string &status)
{
    HSCVolDetail volDetail;
    for (int32_t retryTimes = 0; retryTimes <= WAIT_TIMES; retryTimes++) {
        if (!GetVolDetails(volObj, volDetail)) {
            ERRLOG("Get volume(%s) details failed, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
            return FAILED;
        }
        if (volDetail.m_hostVolume.m_status == status) {
            DBGLOG("Volume(%s) is %s status, %s", volObj.m_uuid.c_str(), status.c_str(), m_taskId.c_str());
            return SUCCESS;
        }
        sleep(WAIT_INTERVAL);
    }
    return FAILED;
}

int32_t HCSProtectEngine::AttachVolume(const VolInfo &volObj)
{
    int ret = AttachVolumeHandle(volObj);
    TP_START("TP_HCS_AttachVolumeFailed", 1, &ret);
    TP_END
    if (ret != SUCCESS) {
        m_reportArgs = { volObj.m_uuid.c_str() };
        m_reportParam = {
            "virtual_plugin_restore_job_attach_volume_failed_label",
            JobLogLevel::TASK_LOG_ERROR,
            SubJobStatus::FAILED, 0, 0 };
        return ret;
    }
    m_reportArgs = { m_jobHandle->GetApp().id };
    m_reportParam = {
        "virtual_plugin_restore_job_attach_volume_success_label",
        JobLogLevel::TASK_LOG_INFO,
        SubJobStatus::RUNNING, 0, 0 };
    return SUCCESS;
}

int32_t HCSProtectEngine::AttachVolumeHandle(const VolInfo &volObj)
{
    HSCVolDetail volDetail;
    if (!GetVolDetails(volObj, volDetail)) {
        return FAILED;
    }
    // 若卷已挂载，判断卷是否已挂载在指定的虚拟机上
    if (volDetail.m_hostVolume.m_status == VOLUME_STATUS_INUSE) {
        for (auto &attachPoint : volDetail.m_hostVolume.m_attachPoints) {
            if (m_jobHandle->GetApp().id == attachPoint.m_serverId) {
                INFOLOG("Volume has been attached, volId: %s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
                return SUCCESS;
            }
        }
        ERRLOG("Volume has been attached to another server, volId: %s. %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    if (CheckAttachServerVolume(volObj) != SUCCESS) {
        ERRLOG("Attach server volume failed, volId=%s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    // 查询卷的状态是否为已挂载
    if (CheckVolumeStatus(volObj, VOLUME_STATUS_INUSE) != SUCCESS) {
        ERRLOG("Attach server volume failed, volId=%s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    DBGLOG("Volume(%s) attach success, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
    return SUCCESS;
}

bool HCSProtectEngine::GetVMDetails(const VMInfo &vmInfo, ServerDetail &serverDetail)
{
    GetServerDetailsRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }
    request.SetServerId(vmInfo.m_uuid);
    std::shared_ptr<GetServerDetailsResponse> response = m_ecsClient.GetServerDetails(request);

    if (response == nullptr || !response->Success() ||
        response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK)) {
        ERRLOG("Send get server details request failed, vmId: %s, %s", vmInfo.m_uuid.c_str(), m_taskId.c_str());
        return false;
    }
    serverDetail = response->GetServerDetails();
    return true;
}

int32_t HCSProtectEngine::PowerOnMachineHandle(const VMInfo &vmInfo)
{
    ServerDetail serverDetail;
    if (!GetVMDetails(vmInfo, serverDetail)) {
        return FAILED;
    }
    if (serverDetail.m_hostServerInfo.m_status == SERVER_STATUS_ACTIVE) {
        return SUCCESS;
    }
    PowerOnServerRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    request.SetServerId(vmInfo.m_uuid);
    std::shared_ptr<PowerOnServerResponse> response = m_ecsClient.PowerOnServer(request);
    if (response == nullptr) {
        ERRLOG("Send power on server request failed, vmId: %s. %s", vmInfo.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::ACCEPTED) &&
        response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::NO_CONTENT)) {
            ERRLOG("Power on server failed, vmId: %s, statusCode: %u, %s", vmInfo.m_uuid.c_str(),
                response->GetStatusCode(), m_taskId.c_str());
            return FAILED;
    }
    // 查询虚拟机状态是否已上电
    for (int32_t retryTimes = 0; retryTimes <= WAIT_TIMES; retryTimes++) {
        if (!GetVMDetails(vmInfo, serverDetail)) {
            return FAILED;
        }
        if (serverDetail.m_hostServerInfo.m_status == SERVER_STATUS_ACTIVE) {
            DBGLOG("Power on machine(%s, %s) success, %s", vmInfo.m_name.c_str(), vmInfo.m_uuid.c_str(),
                m_taskId.c_str());
            return SUCCESS;
        }
        sleep(WAIT_INTERVAL);
    }
    ERRLOG("Power on server failed after query status, vmId: %s, %s", vmInfo.m_uuid.c_str(), m_taskId.c_str());
    return FAILED;
}

int32_t HCSProtectEngine::PowerOffMachine(const VMInfo &vmInfo)
{
    int ret = PowerOffMachineHandle(vmInfo);
    m_reportArgs = { vmInfo.m_uuid.c_str() };
    if (ret != SUCCESS) {
        m_reportParam = {
            "virtual_plugin_restore_job_power_off_vm_failed_label",
            JobLogLevel::TASK_LOG_ERROR,
            SubJobStatus::FAILED, 0, 0 };
        return ret;
    }
    m_reportParam = {
        "virtual_plugin_restore_job_power_off_vm_success_label",
        JobLogLevel::TASK_LOG_INFO,
        SubJobStatus::RUNNING, 0, 0 };
    return SUCCESS;
}

int32_t HCSProtectEngine::PowerOffMachineHandle(const VMInfo &vmInfo)
{
    ServerDetail serverDetail;
    if (!GetVMDetails(vmInfo, serverDetail)) {
        return FAILED;
    }
    if (serverDetail.m_hostServerInfo.m_status == SERVER_STATUS_SHUTOFF) {
        return SUCCESS;
    }
    PowerOffServerRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    request.SetServerId(vmInfo.m_uuid);
    std::shared_ptr<PowerOffServerResponse> response = m_ecsClient.PowerOffServer(request);
    if (response == nullptr) {
        ERRLOG("Send power off server request failed, vmId: %s. %s", vmInfo.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::ACCEPTED) &&
        response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::NO_CONTENT)) {
            ERRLOG("Power off server failed, vmId: %s, statusCode: %u, %s", vmInfo.m_uuid.c_str(),
                response->GetStatusCode(), m_taskId.c_str());
            return FAILED;
    }
    // 查询虚拟机状态是否已下电
    for (int32_t retryTimes = 0; retryTimes <= WAIT_TIMES; retryTimes++) {
        if (!GetVMDetails(vmInfo, serverDetail)) {
            return FAILED;
        }
        if (serverDetail.m_hostServerInfo.m_status == SERVER_STATUS_SHUTOFF) {
            DBGLOG("Power off machine(%s) id(%s) success, %s", vmInfo.m_name.c_str(), vmInfo.m_uuid.c_str(),
                m_taskId.c_str());
            return SUCCESS;
        }
        sleep(WAIT_INTERVAL);
    }
    ERRLOG("Power off server failed after query status, vmId: %s. %s", vmInfo.m_uuid.c_str(), m_taskId.c_str());
    return FAILED;
}

int32_t HCSProtectEngine::AllowBackupInLocalNode(const AppProtect::BackupJob& job, int32_t &errorCode)
{
    m_appEnv = job.protectEnv;
    m_backupPara = std::make_shared<AppProtect::BackupJob>(job);
    if (m_backupPara == nullptr) {
        ERRLOG("Get backup job parameter failed, %s", m_taskId.c_str());
        return FAILED;
    }
    m_application = m_backupPara->protectObject;
    if (CheckEnvConnection(job.protectEnv, job.protectObject.id, errorCode) != SUCCESS) {
        ERRLOG("Check env connection failed;");
        return FAILED;
    }
    std::shared_ptr<JobCommonInfo> jobCommonInfoPtr = std::make_shared<JobCommonInfo>(m_backupPara);
    std::shared_ptr<JobHandle> jobHandle = std::make_shared<JobHandle>(JobType::BACKUP, jobCommonInfoPtr);
    SetJobHandle(jobHandle);
    if (!DoGetMachineMetadata()) {
        ERRLOG("Get mechine metadata failed, %s", m_taskId.c_str());
        return FAILED;
    }
    std::string volId;
    VolInfo volumeInfo;
    if (m_backupPara->protectSubObject.size() == 0) {
        volId = m_vmInfo.m_volList[0].m_uuid; // ECS任意一个卷ID
    } else {
        volId = m_backupPara->protectSubObject[0].id; // 已保护磁盘的任一磁盘id
    }
    for (const VolInfo &vol : m_vmInfo.m_volList) {
        if (vol.m_uuid == volId) {
            volumeInfo = vol;
            break;
        }
    }
    std::string authExtendInfo = job.protectEnv.auth.extendInfo;
    int32_t erro;
    if (CheckStorageConnection(volumeInfo, authExtendInfo, erro) != SUCCESS) {
        if (erro == DSWARE_API_ERR_VBSTOOL_CONF_FAULT) {
            errorCode = CHECK_VBS_CONFIG_FAILED;
        } else if (erro == NO_STORAGE_INFO) {
            errorCode = NO_STORAGE_INFO;
        } else {
            errorCode = CHECK_STORAGE_CONNECT_FAILED;
        }
        return FAILED;
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckProtectEnvConn(const AppProtect::ApplicationEnvironment& env, const std::string &vmId,
    int32_t &errorCode)
{
    m_hcsOpServiceUtils.GetAppEnv(env);
    m_appEnv = env;
    GetServerDetailsRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    request.SetServerId(vmId);
    request.SetNeedRetry(true);
    std::shared_ptr<GetServerDetailsResponse> resp = m_ecsClient.GetServerDetails(request);
    if (resp == nullptr) {
        ERRLOG("Check protect env connection failed, %s", m_taskId.c_str());
        errorCode = CHECK_HCS_CONNECT_FAILED;
        return FAILED;
    }
    if (resp->GetStatusCode() == static_cast<uint32_t>(HcsExternalStatusCode::NOT_FOUND) &&
        resp->GetBody().find("itemNotFound") != std::string::npos) {
        ERRLOG("The vm(%s) is not found.", vmId.c_str());
        errorCode = VM_NOT_EXSIT;
        return FAILED;
    }
    if (resp->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK)) {
        ERRLOG("Failed to get vm(%s) details.", vmId.c_str());
        errorCode = CHECK_HCS_CONNECT_FAILED;
        return FAILED;
    }
    DBGLOG("Product env(%s) is reachable, %s", vmId.c_str(), m_taskId.c_str());
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckStorageEnvConn(const ApplicationEnvironment &appEnv)
{
    std::string backupMode;
    if (GetHcsBackupMode(backupMode) == SUCCESS && backupMode == "VBS") {
        INFOLOG("Jump check storage connection when using vbs mode.");
        return SUCCESS;
    }
    DBGLOG("Start to check storage environment connection.");
    HcsResourceAccess hcsResourceAccess(appEnv);
    AppProtect::ApplicationEnvironment returnEnv;
    if (hcsResourceAccess.CheckStorageConnect(returnEnv) != SUCCESS) {
        ERRLOG("Failed to check storage connection in iSCSI mode.");
        return FAILED;
    }
    DBGLOG("Connect to storage success.");
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckEnvConnection(
    const AppProtect::ApplicationEnvironment &env, const std::string &vmId, int32_t &errorCode)
{
    if (CheckProtectEnvConn(env, vmId, errorCode) != SUCCESS) {
        ERRLOG("Failed to check protect environment.");
        CheckCertIsExist(errorCode);
        return FAILED;
    }
    return SUCCESS;
}

void HCSProtectEngine::CheckCertIsExist(int32_t &errorCode)
{
    if (m_cinderCertMgr->IsVerifyCert() && m_cinderCertMgr->GetCertPath() == "") {
        ERRLOG("The cinder certification is not exists.");
        errorCode = HCS_CERT_NOT_EXIST;
        return;
    }
    return;
}

void HCSProtectEngine::SetErrorCodeParam(const int32_t errorCode, std::vector<std::string> &certParams)
{
    if (errorCode == HCS_CERT_NOT_EXIST) {
        std::string param;
        if (!m_certCpsIsExists) {
            param = CPS_CERT;
        }
        certParams.push_back(param);
        INFOLOG("errorCode: %d, certParams: %s", errorCode, param.c_str());
    }
    return;
}

int32_t HCSProtectEngine::AllowRestoreInLocalNode(const AppProtect::RestoreJob &job, int32_t &errorCode)
{
    m_appEnv = job.targetEnv;
    m_restorePara = std::make_shared<AppProtect::RestoreJob>(job);
    if (m_restorePara == nullptr) {
        ERRLOG("Get restore job parameter failed, %s", m_taskId.c_str());
        return false;
    }
    m_application = m_restorePara->targetObject;
    return CheckEnvConnection(job.targetEnv, job.targetObject.id, errorCode);
}

int32_t HCSProtectEngine::AllowBackupSubJobInLocalNode(const AppProtect::BackupJob& job,
    const AppProtect::SubJob& subJob, int32_t &errorCode)
{
    m_appEnv = job.protectEnv;
    m_backupPara = std::make_shared<AppProtect::BackupJob>(job);
    if (m_backupPara == nullptr) {
        ERRLOG("Get backup job parameter failed, %s", m_taskId.c_str());
        return FAILED;
    }
    m_application = m_backupPara->protectObject;
    if (CheckProtectEnvConn(job.protectEnv, job.protectObject.id, errorCode) != SUCCESS) {
        CheckCertIsExist(errorCode);
        return FAILED;
    }
    if (subJob.jobName == REPORT_COPY_SUB_JOB) {
        return SUCCESS;
    }
    if (subJob.jobType == SubJobType::type::POST_SUB_JOB) {
        return SUCCESS;
    }
    BackupSubJobInfo backupSubJob {};
    if (!Module::JsonHelper::JsonStringToStruct(subJob.jobInfo, backupSubJob)) {
        ERRLOG("Get backup subjob info failed, %s", m_taskId.c_str());
        return FAILED;
    }
    std::string authExtendInfo = job.protectEnv.auth.extendInfo;
    int32_t erro;
    m_backupPara = std::make_shared<AppProtect::BackupJob>(job);
    std::shared_ptr<JobCommonInfo> jobCommonInfoPtr = std::make_shared<JobCommonInfo>(m_backupPara);
    std::shared_ptr<JobHandle> jobHandle = std::make_shared<JobHandle>(JobType::BACKUP, jobCommonInfoPtr);
    SetJobHandle(jobHandle);
    if (CheckStorageConnection(backupSubJob.m_volInfo, authExtendInfo, erro) != SUCCESS) {
        if (erro == DSWARE_API_ERR_VBSTOOL_CONF_FAULT) {
            errorCode = CHECK_VBS_CONFIG_FAILED;
        } else if (erro == NO_STORAGE_INFO) {
            errorCode = NO_STORAGE_INFO;
        } else {
            errorCode = CHECK_STORAGE_CONNECT_FAILED;
        }
        return FAILED;
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::AllowRestoreSubJobInLocalNode(const AppProtect::RestoreJob& job,
    const AppProtect::SubJob& subJob, int32_t &errorCode)
{
    m_appEnv = job.targetEnv;
    m_restorePara = std::make_shared<AppProtect::RestoreJob>(job);
    if (m_restorePara == nullptr) {
        ERRLOG("Get restore job parameter failed, %s", m_taskId.c_str());
        return false;
    }
    m_application = m_restorePara->targetObject;
    if (CheckProtectEnvConn(job.targetEnv, job.targetObject.id, errorCode) != SUCCESS) {
        CheckCertIsExist(errorCode);
        return FAILED;
    }
    if (subJob.jobType == SubJobType::type::POST_SUB_JOB) {
        return SUCCESS;
    }
    if (subJob.jobName == "PostSubJob") {
        return SUCCESS;
    }
    SubJobExtendInfo subJobExtendInfo {};
    if (!Module::JsonHelper::JsonStringToStruct(subJob.jobInfo, subJobExtendInfo)) {
        ERRLOG("Get restore subjob info failed, %s.", m_taskId.c_str());
        return FAILED;
    }
    VolInfo volInfo {};
    if (!Module::JsonHelper::JsonStringToStruct(subJobExtendInfo.m_targetVolumeInfo, volInfo)) {
        ERRLOG("Failed to get vol info, %s", m_taskId.c_str());
        return FAILED;
    }
    std::string authExtendInfo = job.targetEnv.auth.extendInfo;
    int32_t erro;
    m_restorePara = std::make_shared<AppProtect::RestoreJob>(job);
    std::shared_ptr<JobCommonInfo> jobCommonInfoPtr = std::make_shared<JobCommonInfo>(m_restorePara);
    std::shared_ptr<JobHandle> jobHandle = std::make_shared<JobHandle>(JobType::RESTORE, jobCommonInfoPtr);
    SetJobHandle(jobHandle);
    if (CheckStorageConnection(volInfo, authExtendInfo, erro) != SUCCESS) {
        if (erro == DSWARE_API_ERR_VBSTOOL_CONF_FAULT) {
            errorCode = CHECK_VBS_CONFIG_FAILED;
        } else if (erro == NO_STORAGE_INFO) {
            errorCode = NO_STORAGE_INFO;
        } else {
            errorCode = CHECK_STORAGE_CONNECT_FAILED;
        }
        return FAILED;
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckStorageConnection(const VolInfo &volInfo, const std::string &authExtendInfo,
    int32_t &erroCode)
{
    std::shared_ptr<VolumeHandler> storageVolHandler = nullptr;
    m_noNeedInitOceanVolHandler = true;
    int ret = GetVolumeHandler(volInfo, storageVolHandler);
    if (ret != SUCCESS) {
        ERRLOG("Get volume handler failed, datastore type:%s, %s", volInfo.m_datastore.m_type.c_str(),
            m_taskId.c_str());
        erroCode = ret;
        return FAILED;
    }
    if (storageVolHandler->TestDeviceConnection(authExtendInfo, erroCode) != SUCCESS) {
        ERRLOG("Connecte to storage failed, %s", m_taskId.c_str());
        return FAILED;
    }
    INFOLOG("Connecte to storage success, %s", m_taskId.c_str());
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckBackupJobType(const VirtPlugin::JobTypeParam &jobTypeParam, bool &checkRet)
{
    if (CheckPreSnapshotExist(jobTypeParam) != SUCCESS) {
        ERRLOG("Presnapshot not exist, turn to full backup.");
        checkRet = false;
        return SUCCESS;
    }
    // 获得当前卷备份的存储IP或SN
    std::string storageIp = jobTypeParam.m_snapshotInfo.m_volSnapList.back().m_datastore.m_ip;
    std::string storageSn = jobTypeParam.m_snapshotInfo.m_volSnapList.back().m_datastore.m_moRef;
    if (storageIp.empty() && storageSn.empty()) {
        ERRLOG("Failed to get storage from snapshot list, task ID: %s", jobTypeParam.m_job.jobId.c_str());
        return FAILED;
    }
    // 解析存储信息
    std::vector<StorageInfo> storageVector;
    if (!Utils::GetStoragesInfoFromExtendInfo(jobTypeParam.m_job.protectEnv.auth.extendInfo, storageVector)) {
        ERRLOG("Failed to get storageinfo from extendinfo.");
        return FAILED;
    }
    // 获得存储类型，分布式存储提供的SN号就是其IP，检查IP是否一致，集中式检查SN号
    std::string storageType;
    for (const auto &item : storageVector) {
        if (item.m_sn == storageSn) {
            storageType = item.m_storageType;
            break;
        }
        if (item.m_sn == storageIp && item.m_storageType == STORAGE_TYPE_DISTRIBUTED) {
            storageType = STORAGE_TYPE_DISTRIBUTED;
            break;
        }
    }
    // 8.1.2以前的分布式存储，备份模式为iscsi时，增量转全量
    std::string backupMode;
    if (storageType == STORAGE_TYPE_DISTRIBUTED && GetHcsBackupMode(backupMode) == SUCCESS && backupMode == "ISCSI") {
        if (CheckDistributeVersion(storageVector, jobTypeParam) != SUCCESS) {
            ERRLOG("Pacific version dont allow increase backup.");
            checkRet = false;
            return SUCCESS;
        }
    }

    checkRet = true;
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckDistributeVersion(std::vector<StorageInfo> &storageVector,
    const VirtPlugin::JobTypeParam &jobTypeParam)
{
    bool isBackUp = true;
    std::map<std::string, int32_t> storageCheckMap;    // 避免同一存储多次检查
    for (const auto &snap : jobTypeParam.m_snapshotInfo.m_volSnapList) {
        std::string storageIp = snap.m_datastore.m_ip;
        std::string storageSn = snap.m_datastore.m_dcMoRef;
        std::string poolId = snap.m_datastore.m_poolId;
        if (storageCheckMap.find(storageIp) != storageCheckMap.end()) {
            continue;
        }
        storageCheckMap[storageIp] = 1;
        for (const StorageInfo &item : storageVector) {
            if (item.m_ip != storageIp && item.m_ipList.find(storageIp) == std::string::npos) {
                continue;
            }
            VolInfo vol;
            vol.m_datastore.m_ip = storageIp;
            int32_t version;
            std::string err;
            std::shared_ptr<FusionStorageRestApiOperator> restApi =
                std::make_shared<FusionStorageRestApiOperator>(storageIp, poolId, m_jobId, m_subJobId);
            if (restApi == nullptr) {
                ERRLOG("Null restApi ptr.");
                return FAILED;
            }
            restApi->SetOpService(m_hcsOpServiceUtils.GetIsOpServiceEnv());
            restApi->InitRestApiOperator();
            restApi->GetStorageInfo(jobTypeParam.m_job.protectEnv.auth.extendInfo, vol, isBackUp);
            if (restApi->GetVersion(version, err) != SUCCESS) {
                ERRLOG("Get pacific version failed.");
                return FAILED;
            }
            if (version < MIN_PACIFIC_VERSION_INCREASE) {
                ERRLOG("Pacific version %d is lower than 813.", version);
                return FAILED;
            }
            if (item.m_vbsIp.empty()) {
                ERRLOG("Dont find vbs node info.");
                return FAILED;
            }
            break;
        }
    }
    INFOLOG("Check pacific version success.");
    return SUCCESS;
}

void HCSProtectEngine::DiscoverApplications(std::vector<Application>& returnValue, const std::string& appType)
{
    return;
}

void HCSProtectEngine::CheckApplication(ActionResult &returnValue, const ApplicationEnvironment &appEnv,
    const AppProtect::Application &application)
{
    // VDC资源接入连通性检查证书内容为空，无需证书解析
    HcsResourceAccess m_hcsResourceAccess(appEnv, application);
    int32_t ret = m_hcsResourceAccess.CheckAppConnect(returnValue);
    if (ret != SUCCESS) {
        ERRLOG("Failed to check hcs connectivity, %s", m_taskId.c_str());
        return;
    }

    CheckApplicationReturnValue(returnValue, appEnv, application);
    return;
}

void HCSProtectEngine::ListApplicationResource(std::vector<ApplicationResource>& returnValue,
    const ApplicationEnvironment& appEnv, const Application& application, const ApplicationResource& parentResource)
{
    return;
}

// Agent 通过抛异常判扫描成功失败，
void HCSProtectEngine::ListApplicationResourceV2(ResourceResultByPage& page, const ListResourceRequest& request)
{
    Json::Reader conditionReader;
    Json::Value conditionJson;
    if (!conditionReader.parse(request.condition.conditions, conditionJson)) {
        ERRLOG("Failed to read condition, %s", m_taskId.c_str());
        return;
    }
    if (!conditionJson.isMember(RESOURCE_TYPE_KEY)) {
        ERRLOG("No source key sourceType, %s", m_taskId.c_str());
        return;
    }
    std::string sourceType = conditionJson[RESOURCE_TYPE_KEY].asString();
    HcsResourceAccess hcsResourceAccess(request.appEnv, request.condition);
    std::string errMessage;
    std::vector<std::string> erroParam;
    if (sourceType == RESOURCE_TYPE_TENANT) {
        if (hcsResourceAccess.GetTenantInfo(page) != SUCCESS) {
            errMessage = "User name or password error.";
            ERRLOG("Failed to get tenant info, %s", m_taskId.c_str());
            ThrowPluginException(CHECK_HCS_TENANT_FAILED, errMessage);
        }
    } else if (sourceType == RESOURCE_TYPE_PROJECT) {
        std::string vdcUserName;
        if (GetProjectResource(page, hcsResourceAccess, request, vdcUserName) != SUCCESS) {
            errMessage = "Failed to get project by vdc.";
            erroParam.push_back(vdcUserName);
            ERRLOG("Failed to get project resource, %s", m_taskId.c_str());
            ThrowPluginException(QUERY_PROJECTS_LIST_FAILED, errMessage, erroParam);
        }
    } else if (sourceType == RESOURCE_TYPE_VM) {
        if (GetVirtualMachineResource(page, hcsResourceAccess, request) != SUCCESS) {
            errMessage = "Scan cloud server failed.";
            ERRLOG("Failed to get virtual machine resource, %s", m_taskId.c_str());
            ThrowPluginException(SCAN_CLOUD_SEVER_FAILED, errMessage);
        }
    } else if (sourceType == RESOURCE_TYPE_DISK) {
        if (GetDiskResourceDetail(page, hcsResourceAccess, request) != SUCCESS) {
            errMessage = "Scan disk failed.";
            ERRLOG("Failed to get project resource, %s", m_taskId.c_str());
            ThrowPluginException(SCAN_CLOUD_SEVER_FAILED, errMessage);
        }
    } else if (sourceType == RESOURCE_TYPE_VDC) {
        if (GetVDCResource(page, hcsResourceAccess, request) != SUCCESS) {
            ERRLOG("Failed to get VDC resource, %s", m_taskId.c_str());
        }
    } else {
        ERRLOG("Not found the resource type: %s, %s", sourceType.c_str(), m_taskId.c_str());
    }
    return;
}

int32_t HCSProtectEngine::GetHcsBackupMode(std::string &backupMode)
{
    backupMode = Module::ConfigReader::getString("HcsConfig", "FusionStorageApiMode");
    if (backupMode == "ISCSI") {
        return SUCCESS;
    }
    std::string deployScence;
    int32_t ret = Utils::GetAgentDeployScence(deployScence);
    if (ret != SUCCESS) {
        WARNLOG("Failed to get the agent deploy scence.");
        return FAILED;
    }
    if (deployScence == INTERNAL_BACKUP_SCENCE) {
        backupMode = "ISCSI";
    }

    return SUCCESS;
}

void HCSProtectEngine::DiscoverHostCluster(ApplicationEnvironment& returnEnv, const ApplicationEnvironment& appEnv)
{
    return;
}

void HCSProtectEngine::DiscoverAppCluster(ApplicationEnvironment& returnEnv, const ApplicationEnvironment& appEnv,
    const Application& application)
{
    m_hcsOpServiceUtils.GetAppEnv(appEnv);
    HcsResourceAccess m_hcsResourceAccess(appEnv);
    if (m_hcsResourceAccess.CheckStorageConnect(returnEnv) != SUCCESS) {
        ERRLOG("CheckStorageConnect failed, %s", m_taskId.c_str());
    }
    return;
}

bool HCSProtectEngine::DoGetMachineMetadata()
{
    int32_t ret = SUCCESS;
    TP_START("TP_HCS_GetMachineMetadataFailed", 1, &ret);
    TP_END
    if (ret == FAILED) {
        return false;
    }
    if (m_machineMetaCached) {
        DBGLOG("Machine metadata cached, %s", m_taskId.c_str());
        return true;
    }

    GetServerDetailsRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }
    DBGLOG("Get machine metadata, server id: %s, %s", m_backupPara->protectObject.id.c_str(), m_taskId.c_str());
    request.SetServerId(m_backupPara->protectObject.id);

    std::shared_ptr<GetServerDetailsResponse> response = m_ecsClient.GetServerDetails(request);
    if (response == nullptr || response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK)) {
        ERRLOG("Get server details failed, %s", m_taskId.c_str());
        return false;
    }
    if (!TransServerDetail2VMInfo(response->GetServerDetails(), m_vmInfo)) {
        ERRLOG("Translate ServerDetail to VMInfo failed, %s", m_taskId.c_str());
        return false;
    }
    if (m_vmInfo.m_volList.size() == 0) {
        ERRLOG("Vm volume size is 0.");
        return false;
    }
    m_machineMetaCached = true;
    DBGLOG("Do get machine metadata success, %s", m_taskId.c_str());
    return true;
}

double HCSProtectEngine::GetMinAvailableCapacity()
{
    std::string threshold;
    VirtPlugin::Utils::GetStringValueByKeyFromJsonString(
        m_backupPara->extendInfo, VirtPlugin::AVAILABLE_CAPACITY_THRESHOLD_KEY, threshold);
    if (threshold == "") {
        return VirtPlugin::DEFAULT_STORAGE_THRESHOLD_LIMIT * PERCENTAGE_ONE;
    }
    double result = Utils::SafeStod(threshold);
    if (result < VirtPlugin::MIN_STORAGE_THRESHOLD_LIMIT || result > VirtPlugin::MAX_STORAGE_THRESHOLD_LIMIT) {
        return VirtPlugin::DEFAULT_STORAGE_THRESHOLD_LIMIT * PERCENTAGE_ONE;
    }
    return result * PERCENTAGE_ONE;
}

int32_t HCSProtectEngine::CheckBeforeCreateSnapshot(const std::vector<VolInfo> &volList)
{
    std::string  noCheck = Module::ConfigReader::getString("HcsConfig", "IgnoreSnapshotPreCheck");
    if (noCheck == "true") {
        INFOLOG("Dont check storage used capacity.");
        return SUCCESS;
    }
    double usedRate;
    double maxUsedRate = 1.0 - GetMinAvailableCapacity();
    INFOLOG("Max used rate is %f.", maxUsedRate);
    std::set<std::string> storageCheck;    // 有多个卷需要打快照时，避免同一存储池多次检查
    std::shared_ptr<VolumeHandler> storageVolHandler = nullptr;
    for (const VolInfo &vol : volList) {
        if (!storageCheck.insert(vol.m_datastore.m_poolId).second) {
            continue;
        }
        if (GetVolumeHandler(vol, storageVolHandler) != SUCCESS) {
            ERRLOG("Get volume handle failed.");
            return FAILED;
        }
        storageVolHandler->QueryStoragePoolUsedRate(usedRate);
        if (usedRate > maxUsedRate) {
            ERRLOG("Storage pool usage %f is over limit(%f).", usedRate, maxUsedRate);
            ApplicationLabelType resourceCheckLabel;
            resourceCheckLabel.level = JobLogLevel::TASK_LOG_ERROR;
            resourceCheckLabel.label = CHECK_STORAGE_USAGE_FAILED_LABEL;
            resourceCheckLabel.params = std::vector<std::string>{vol.m_datastore.m_poolId,
                to_string(int(maxUsedRate/PERCENTAGE_ONE))};
            ReportJobDetail(resourceCheckLabel);
            return FAILED;
        }
    }
    INFOLOG("Storage pool usage check success.");
    return SUCCESS;
}

bool HCSProtectEngine::DoCreateSnapshot(const std::vector<VolInfo> &volList,
    SnapshotInfo &snapshot, std::string &errCode)
{
    {
        // 控制多任务竞争
        std::lock_guard<std::mutex> lock(m_mutex);
        int32_t createSnapshotMaxLimit = Module::ConfigReader::getInt(
            std::string("HcsConfig"), std::string("CreateSnapshotLimit"));
        DBGLOG("Start adding the snapshot creation task, current snapshot value[%d]. %s",
            g_curCreateSnapshotValue.load(), m_taskId.c_str());
        while (g_curCreateSnapshotValue > createSnapshotMaxLimit) {
            std::this_thread::sleep_for(std::chrono::seconds(CREATE_SNAPSHOT_WAIT_TIME));
        }
        g_curCreateSnapshotValue += volList.size();
    }
    int32_t remainVolListSize = volList.size();
    for (const auto &vol : volList) {
        VolSnapInfo volSnap;
        bool useEvs = false;
        int32_t retryTimes = Module::ConfigReader::getInt(
            std::string("HcsConfig"), std::string("CreateSnapshotApigwFailedRetry"));
        while (retryTimes > 0 && !CreateVolumeSnapshot(vol, volSnap, errCode, useEvs)) {
            if (errCode != "APIGW.0308" && errCode != "403") {
                g_curCreateSnapshotValue -= remainVolListSize;
                ERRLOG("Create volume snapshot failed. Volume id: %s, %s, current snapshot value[%d]",
                    vol.m_uuid.c_str(), m_taskId.c_str(), g_curCreateSnapshotValue.load());
                return false;
            }
            useEvs = true;
            ERRLOG("Create volume snapshot failed. Volume id: %s, %s, retryTimes: %d",
                vol.m_uuid.c_str(), m_taskId.c_str(), retryTimes);
            uint64_t randNum = 0;
            if (Utils::GetRandom(randNum) != SUCCESS) {
                randNum = CREATE_SNAPSHOT_RETRY_WAIT_TIME;
            }
            std::this_thread::sleep_for(std::chrono::seconds(CREATE_SNAPSHOT_RETRY_WAIT_TIME +
                randNum % CREATE_SNAPSHOT_RETRY_WAIT_TIME));
            retryTimes--;
        }

        if (retryTimes == 0) {
            g_curCreateSnapshotValue -= remainVolListSize;
            ERRLOG("Create volume snapshot failed. Volume id: %s, %s, current snapshot value[%d]", vol.m_uuid.c_str(),
                m_taskId.c_str(), g_curCreateSnapshotValue.load());
            return false;
        }

        if (vol.m_type != HCS_VOLTYPE_SHARED) {
            snapshot.m_volSnapList.push_back(volSnap);
        }
        --g_curCreateSnapshotValue;
        --remainVolListSize;
    }
    return true;
}

bool HCSProtectEngine::CreateVolumeSnapshot(const VolInfo &volInfo, VolSnapInfo &volSnap, std::string &errCode,
    bool useEvs)
{
    if (volInfo.m_type == HCS_VOLTYPE_SHARED) {
        WARNLOG("Skipping shared volume. %s", m_taskId.c_str());
        return true;
    }

    HCSVolumeDSExtendInfo dsExtend;
    if (!Module::JsonHelper::JsonStringToStruct(volInfo.m_datastore.m_extendInfo, dsExtend)) {
        ERRLOG("Trans datastore extend to struct failed. Extend: %s, %s",
            WIPE_SENSITIVE(volInfo.m_datastore.m_extendInfo).c_str(), m_taskId.c_str());
        return false;
    }

    CreateSnapshotRequestBodyMsg body;
    FillSnapshotRequestBody(body, GenerateSnapshotName(dsExtend.m_volId), volInfo);
    CreateSnapshotRequest request;
    request.SetBody(body);
    request.SetApiType(useEvs ? ApiType::EVS : ApiType::CINDER);
    if (!FormHttpRequest(request, false)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }
    std::shared_ptr<CreateSnapshotResponse> response = m_cinderClient->CreateSnapshot(request);
    if (response == nullptr) {
        ERRLOG("Create volume snapshot failed, %s", m_taskId.c_str());
        return false;
    }
    if (CertErrorCode.find(response->GetErrCode()) != CertErrorCode.end()) {
        ERRLOG("Check hcs cert failed.");
        errCode = "1677931024";
        return false;
    }
    ParseCreateSnapResponse(response, errCode);
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::ACCEPTED)) {
        ERRLOG("Create snapshot failed. Return code: %u, %s", response->GetStatusCode(), m_taskId.c_str());
        return false;
    }
    if (!response->Serial()) {
        ERRLOG("Create snapshot response serial failed, %s", m_taskId.c_str());
        return false;
    }
    if (response->GetSnapshotDetails().m_snapshotDetails.m_status == SNAPSHOT_STATUS_ERROR) {
        ERRLOG("Create snapshot failed, snapshot status is error, %s", m_taskId.c_str());
        volSnap.m_snapshotId = response->GetSnapshotDetails().m_snapshotDetails.m_id;
        // 若本次删除失败，可以等待下次全量备份的后置任务，继续重试删除
        if (!DeleteVolumeSnapshot(volSnap)) {
            ERRLOG("Delete volume snapshot failed, volId: %s, snapId: %s, taskId: %s", volInfo.m_uuid.c_str(),
                volSnap.m_snapshotId.c_str(), m_taskId.c_str());
        }
        return false;
    }
    return FillUpVolSnapInfo(volInfo, response->GetSnapshotDetails().m_snapshotDetails, "", volSnap);
}

void HCSProtectEngine::FillSnapshotRequestBody(CreateSnapshotRequestBodyMsg &body, const std::string &snapName,
    const VolInfo &volInfo)
{
    body.m_snapshot.m_name = snapName;
    body.m_snapshot.m_volumeId = volInfo.m_uuid;
    body.m_snapshot.m_description = m_snapDescription;
    body.m_snapshot.m_force = "true";
    body.m_snapshot.m_snapshotMetadata.m_enableActive = "false";
    if (volInfo.m_datastore.m_type.find(FUSIONSTORAGE_TYPE) != std::string::npos) {
        boost::posix_time::ptime nycTime = boost::posix_time::microsec_clock::universal_time();
        boost::posix_time::ptime timeEpoch(boost::gregorian::date(STANDARD_START_TIME_1970, 1, 1));
        boost::posix_time::time_duration diff = nycTime - timeEpoch;
        body.m_snapshot.m_description += "_Time:" + boost::lexical_cast<std::string>(diff.total_microseconds());
    }
}

void HCSProtectEngine::ParseCreateSnapResponse(std::shared_ptr<CreateSnapshotResponse> createSnapRes,
    std::string &errCode)
{
    Json::Value responseJsonBody;
    if (!Module::JsonHelper::JsonStringToJsonValue(createSnapRes->GetBody(), responseJsonBody)) {
        ERRLOG("Convert the snapshot body failed.");
        return;
    }

    if (responseJsonBody.isMember("error_code") && responseJsonBody["error_code"].compare("APIGW.0308") == SUCCESS) {
        ERRLOG("ParseCreateSnapResponse(), response: %s. %s",
            WIPE_SENSITIVE(createSnapRes->GetBody()).c_str(), m_taskId.c_str());
        errCode = "APIGW.0308";
    } else {
        errCode = std::to_string(createSnapRes->GetStatusCode());
    }
    return;
}

bool HCSProtectEngine::ShowVolumeSnapshot(VolSnapInfo &volSnap)
{
    GetSnapshotRequest request;
    if (!FormHttpRequest(request, false)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }

    request.SetSnapshotId(volSnap.m_snapshotId);
    std::shared_ptr<GetSnapshotResponse> response = m_cinderClient->GetSnapshot(request);
    if (response == nullptr) {
        ERRLOG("Show volume snapshot failed, %s", m_taskId.c_str());
        return false;
    }
    if (response->GetStatusCode() == static_cast<uint32_t>(HcsExternalStatusCode::NOT_FOUND)) {
        volSnap.m_deleted = true;
    }

    return true;
}

bool HCSProtectEngine::FillUpVolSnapInfo(const VolInfo &volInfo,
    const SnapshotDetails &snapDetails, const std::string &snapshotStr, VolSnapInfo &volSnap)
{
    int32_t ret = SUCCESS;
    TP_START("TP_HCS_CreateVolumeSnapshotFailed", 1, &ret);
    TP_END
    if (ret == FAILED) {
        return false;
    }
    volSnap.m_volUuid = volInfo.m_uuid;
    volSnap.m_datastore = volInfo.m_datastore;
    volSnap.m_createTime = snapDetails.m_createdAt;
    volSnap.m_snapshotName = snapDetails.m_name;
    volSnap.m_snapshotWwn = snapDetails.m_metadata.m_snapshotWwn;
    volSnap.m_extendInfo = snapshotStr;
    volSnap.m_snapshotDescription = snapDetails.m_description;
    SnapshotProviderAuth snapProviderAuth;
    if (!Module::JsonHelper::JsonStringToStruct(snapDetails.m_storageProviderAuth, snapProviderAuth)) {
        ERRLOG("Get snap provider auth failed, %s", m_taskId.c_str());
        return false;
    }
    if (volSnap.m_datastore.m_type.find(FUSIONSTORAGE_TYPE) != std::string::npos) {
        // fusionstorage: 存储快照名称不等于 snapshot- + hcs快照id
        volSnap.m_snapshotId = std::string("snapshot-") + snapDetails.m_id;
        volSnap.m_storageSnapId = snapProviderAuth.m_snapName;
    } else if (volSnap.m_datastore.m_type.find(OCEANSTORE_TYPE) != std::string::npos ||
        volInfo.m_datastore.m_type.find(DORADO_TYPE) != std::string::npos) {
        volSnap.m_snapshotId = snapDetails.m_id;
        volSnap.m_storageSnapId = snapProviderAuth.m_lunID;
    } else {
        ERRLOG("Storage type identify failed, datastore type: %s, %s", volSnap.m_datastore.m_type.c_str(),
            m_taskId.c_str());
        return false;
    }

    return true;
}

std::string HCSProtectEngine::GetTaskId() const
{
    return "TaskId=" + m_requestId + ".";
}

bool HCSProtectEngine::TransServerDetail2VMInfo(const ServerDetail &serverDetail, VMInfo &vmInfo)
{
    vmInfo.m_moRef = serverDetail.m_hostServerInfo.m_uuid;
    vmInfo.m_uuid = serverDetail.m_hostServerInfo.m_uuid;
    vmInfo.m_name = serverDetail.m_hostServerInfo.m_name;
    vmInfo.m_location = serverDetail.m_hostServerInfo.m_hostId;

    for (const auto &serverVol : serverDetail.m_hostServerInfo.m_osExtendedVolumesvolumesAttached) {
        VolInfo volInfo;
        if (!ShowVolumeDetail(serverVol.m_uuid, volInfo)) {
            ERRLOG("Show volume detail failed, %s", m_taskId.c_str());
            return false;
        }
        if (volInfo.m_type == HCS_VOLTYPE_SHARED) {
            WARNLOG("Skipping shared volume backup in hcs, volume uuid: %s, taskId: %s",
                volInfo.m_uuid.c_str(), m_taskId.c_str());
            continue;
        }
        volInfo.m_vmMoRef = serverDetail.m_hostServerInfo.m_uuid;
        vmInfo.m_volList.push_back(volInfo);
    }

    DBGLOG("Trans ServerDetail to VMInfo success, %s", m_taskId.c_str());
    return true;
}

bool HCSProtectEngine::TransferVolumeInfo(const HSCVolDetail &volDetail, VolInfo &volInfo)
{
    volInfo.m_moRef = volDetail.m_hostVolume.m_id;
    volInfo.m_uuid = volDetail.m_hostVolume.m_id;
    volInfo.m_name = volDetail.m_hostVolume.m_displayName;
    volInfo.m_type = volDetail.m_hostVolume.m_shareable ? HCS_VOLTYPE_SHARED : volDetail.m_hostVolume.m_volumeType;
    volInfo.m_volSizeInBytes = volDetail.m_hostVolume.m_size * GB_SIZE;
    volInfo.m_bootable = volDetail.m_hostVolume.m_bootable;
    DBGLOG("Volume size: %llu", volInfo.m_volSizeInBytes);
    HSCVolDetail volObj = volDetail;
    if (!Module::JsonHelper::StructToJsonString(volObj, volInfo.m_metadata)) {
        ERRLOG("Trans vol detail to string failed, %s", m_taskId.c_str());
        return false;
    }
    return true;
}

bool HCSProtectEngine::TransVolumeDetail2VolInfo(const HSCVolDetail &volDetail, VolInfo &volInfo)
{
    if (!TransferVolumeInfo(volDetail, volInfo)) {
        return false;
    }
    /* datastore */
    OsVolRepDriverData osDriverData;
    if (!Module::JsonHelper::JsonStringToStruct(volDetail.m_hostVolume.m_osVolRepDriverData, osDriverData)) {
        ERRLOG("Get os-volume-replication:driver_data failed, %s", m_taskId.c_str());
        return false;
    }
    volInfo.m_datastore.m_moRef = osDriverData.m_sn;
    volInfo.m_datastore.m_ip = osDriverData.m_ip;
    volInfo.m_datastore.m_poolId = std::to_string(osDriverData.m_pool);
    volInfo.m_datastore.mPoolIdInt = osDriverData.m_pool;
    volInfo.m_datastore.m_volumeName = osDriverData.m_volName;
    volInfo.m_datastore.m_type = volDetail.m_hostVolume.m_metaData.m_storageType;
    HCSVolumeDSExtendInfo dsExtend;
    dsExtend.m_volWwn = volDetail.m_hostVolume.m_metaData.m_lunWWN;
    dsExtend.m_volId = osDriverData.m_lunId;
    dsExtend.m_volName = volDetail.m_hostVolume.m_displayName;
    if (!Module::JsonHelper::StructToJsonString(dsExtend, volInfo.m_datastore.m_extendInfo)) {
        ERRLOG("Trans datastore extend info to string failed, %s", m_taskId.c_str());
        return false;
    }
    HcsVolExtendInfo volExtendInfo;
    volExtendInfo.m_architecture = volDetail.m_hostVolume.m_volImageMetadata.m_arch;
    volExtendInfo.m_encrypted = volDetail.m_hostVolume.m_metaData.m_encrypted;
    volExtendInfo.m_cmkId = volDetail.m_hostVolume.m_metaData.m_cmkId;
    volExtendInfo.m_cipher = volDetail.m_hostVolume.m_encryptionInfo.m_cipher;
    if (volDetail.m_hostVolume.m_metaData.m_hwPassthrough == "true") {
        volExtendInfo.m_attr = "SCSI";
    } else {
        volExtendInfo.m_attr = "VBD";
    }
    volExtendInfo.m_lunWWN = volDetail.m_hostVolume.m_metaData.m_lunWWN;
    volExtendInfo.m_mode = volDetail.m_hostVolume.m_bootable;
    volExtendInfo.m_size = std::to_string(volDetail.m_hostVolume.m_size);
    if (!Module::JsonHelper::StructToJsonString(volExtendInfo, volInfo.m_extendInfo)) {
        ERRLOG("Trans vol extend info to string failed, %s", m_taskId.c_str());
        return false;
    }
    DBGLOG("Trans VolumeDetail to VolInfo success.%s %s %s %s %s Datastore extend: %s. %s", volInfo.m_uuid.c_str(),
        volInfo.m_name.c_str(), volInfo.m_type.c_str(), volInfo.m_datastore.m_ip.c_str(),
        volInfo.m_datastore.m_poolId.c_str(), WIPE_SENSITIVE(volInfo.m_datastore.m_extendInfo).c_str(),
        m_taskId.c_str());

    return true;
}

bool HCSProtectEngine::ShowVolumeDetail(const std::string &volId, VolInfo &volInfo)
{
    ShowVolumeRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }

    request.SetVolumeId(volId);
    std::shared_ptr<ShowVolumeResponse> response = m_evsClient.ShowVolumeDetail(request);
    if (response == nullptr) {
        ERRLOG("Show volume detail failed, %s", m_taskId.c_str());
        return false;
    }
    if (response->GetStatusCode() == static_cast<uint32_t>(HcsExternalStatusCode::OK)) {
        if (!TransVolumeDetail2VolInfo(response->GetHSCVolDetail(), volInfo)) {
            ERRLOG("Trans VolumeDetail to VolInfo failed, %s", m_taskId.c_str());
            return false;
        }
    } else {
        ERRLOG("Show volume detail return failed, code: %d, taskId: %s", response->GetStatusCode(), m_taskId.c_str());
        return false;
    }
    DBGLOG("Show volume detail success, %s", m_taskId.c_str());
    return true;
}

int HCSProtectEngine::GenVolPair(VMInfo &vmObj, const VolInfo &copyVol, const ApplicationResource &targetVol,
    VolMatchPairInfo &volPairs)
{
    Json::Value volInfo;
    if (!Module::JsonHelper::JsonStringToJsonValue(targetVol.extendInfo, volInfo)) {
        return FAILED;
    }
    std::string targetVolsInfo = volInfo["targetVolumes"].asString();
    DBGLOG("TargetVolsInfo %s. %s", targetVolsInfo.c_str(), m_taskId.c_str());
    Json::Value targetVolumeList;
    if (!Module::JsonHelper::JsonStringToJsonValue(targetVolsInfo, targetVolumeList)) {
        ERRLOG("Transfer %s failed, %s", targetVolsInfo.c_str(), m_taskId.c_str());
        return FAILED;
    }
    ShowVolumeRequest request;
    for (const auto &item : targetVolumeList) {
        std::string uuid = item["uuid"].asString();
        if (uuid.empty()) {
            ERRLOG("Uuid is empty for vol %s, %s", copyVol.m_uuid.c_str(), m_taskId.c_str());
            return FAILED;
        }
        request.SetVolumeId(uuid);
        if (!FormHttpRequest(request)) {
            ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
            return FAILED;
        }
        std::shared_ptr<ShowVolumeResponse> response = m_evsClient.ShowVolumeDetail(request);
        if (response == nullptr) {
            ERRLOG("Show volume response failed, %s", m_taskId.c_str());
            return FAILED;
        }
        VolPair restoreVolPair;
        restoreVolPair.m_originVol = copyVol;
        if (!AssemblyTargetVolInfo(restoreVolPair, response)) {
            ERRLOG("Failed to assembly target vol info, %s", m_taskId.c_str());
            return FAILED;
        }
        volPairs.m_volPairList.push_back(restoreVolPair);
        INFOLOG("Copy vol(%s, %s), target vol(%s, %s), %s", copyVol.m_name.c_str(), copyVol.m_uuid.c_str(),
            response->GetName().c_str(), uuid.c_str(), m_taskId.c_str());
    }
    return SUCCESS;
}

bool HCSProtectEngine::AssemblyTargetVolInfo(VolPair &volumePair, const std::shared_ptr<ShowVolumeResponse> &response)
{
    volumePair.m_targetVol.m_uuid = response->GetVolId();
    volumePair.m_targetVol.m_name = response->GetName();
    volumePair.m_targetVol.m_volSizeInBytes = response->GetSize() * GB_SIZE;
    volumePair.m_targetVol.m_datastore.m_type = response->GetDSType();
    volumePair.m_targetVol.m_datastore.m_moRef = response->GetDSSN();
    volumePair.m_targetVol.m_datastore.m_ip = response->GetStorageIp();
    volumePair.m_targetVol.m_datastore.m_poolId = std::to_string(response->GetPoolId());
    volumePair.m_targetVol.m_datastore.m_volumeName = response->GetVolNameOnStorage();
    VolumeDSExtendInfo dsExtInfo;
    dsExtInfo.m_volName = response->GetName();
    dsExtInfo.m_volId = response->GetLUNID();
    dsExtInfo.m_volWwn = response->GetLUNWWN();

    if (!Module::JsonHelper::StructToJsonString(dsExtInfo, volumePair.m_targetVol.m_datastore.m_extendInfo)) {
        ERRLOG("Convert struct to jsonString failed, %s", m_taskId.c_str());
        return false;
    }
    std::vector<Attachments> attachPoints = response->GetAttachPoints();
    if (attachPoints.empty()) {
        ERRLOG("Volume(%s, %s) has no attach points, %s", volumePair.m_targetVol.m_name.c_str(),
            volumePair.m_targetVol.m_uuid.c_str(), m_taskId.c_str());
        return false;
    }
    for (const auto &it : attachPoints) {
        DBGLOG("Volume attach device is %s, %s", it.m_device.c_str(), m_taskId.c_str());
        volumePair.m_targetVol.m_attachPoints.push_back(VolAttachMents(it.m_device));
    }
    return true;
}

bool HCSProtectEngine::CheckVMStatus()
{
    std::string jobType;
    std::string ecsSupportStatus;
    if (m_jobHandle->GetJobType() == JobType::BACKUP) {
        jobType = "backup";
        ecsSupportStatus = "ECSSupportBackupStatus";
        m_appEnv = m_backupPara->protectEnv;
    } else if (m_jobHandle->GetJobType() == JobType::RESTORE) {
        jobType = "restore";
        ecsSupportStatus = "ECSSupportRestoreStatus";
        m_appEnv = m_restorePara->targetEnv;
    } else {
        ERRLOG("check vm status failed, jobtype: %s. %s", m_jobHandle->GetJobType(), m_taskId.c_str());
        return false;
    }

    GetServerDetailsRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed. %s", m_taskId.c_str());
        return false;
    }
    request.SetServerId(m_jobHandle->GetApp().id);
    std::shared_ptr<GetServerDetailsResponse> resp = m_ecsClient.GetServerDetails(request);
    if (resp == nullptr || resp->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK) ||
        resp->GetOsExtstsVmState() == "") {
        ERRLOG("Can not %s. %s", jobType.c_str(), m_taskId.c_str());
        return false;
    }
    if (m_jobHandle->GetJobType() == JobType::RESTORE && resp->GetSoftDeleteFlag() == "true") {
        ERRLOG("Can not restore to soft delete vm.");
        return false;
    }
    std::string supportStatus = Module::ConfigReader::getString("HcsConfig", ecsSupportStatus);
    std::set<std::string> statusList;
    boost::split(statusList, supportStatus, boost::is_any_of(","));
    std::set<std::string>::iterator pos = statusList.find(resp->GetOsExtstsVmState());
    if (pos == statusList.end()) {
        ERRLOG("Can not exec %s for vmstate: %s. %s", jobType.c_str(),
            resp->GetOsExtstsVmState().c_str(), m_taskId.c_str());
        return false;
    }
    DBGLOG("Can exec %s for vmstate: %s. %s", jobType.c_str(), resp->GetServerStatus().c_str(), m_taskId.c_str());
    return true;
}

/**
 * @brief 检查卷状态
 *
 * @param curVolState
 * @return true
 * @return false
 */
bool HCSProtectEngine::CheckVolStatus(const std::string &curVolState)
{
    std::string supportStatus = Module::ConfigReader::getString("HcsConfig", "EVSSupportStatus");
    std::set<std::string> statusList;
    boost::split(statusList, supportStatus, boost::is_any_of(","));
    std::set<std::string>::iterator pos = statusList.find(curVolState);
    if (pos == statusList.end()) {
        ERRLOG("Can not exec restore for vol status: %s, %s", curVolState.c_str(), m_taskId.c_str());
        return false;
    }
    return true;
}

bool HCSProtectEngine::CheckTargetVolume(const VolInfo &copyVolObj, const std::string &targetVolUUID)
{
    ShowVolumeRequest request;
    request.SetVolumeId(targetVolUUID);
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return false;
    }
    std::shared_ptr<ShowVolumeResponse> response = m_evsClient.ShowVolumeDetail(request);
    if (response == nullptr) {
        ERRLOG("Show volume response failed, %s", m_taskId.c_str());
        return false;
    }
    if (!CheckArchitecture(copyVolObj, response->GetArch())) {
        return false;
    }
    if (!CheckVolStatus(response->GetStatus())) {
        return false;
    }
    if (!CheckVolBootMode(copyVolObj.m_bootable, response->GetVolBootable())) {
        return false;
    }
    uint64_t sizeInBytes = response->GetSize() * GB_SIZE;
    if (sizeInBytes < copyVolObj.m_volSizeInBytes) {
        ERRLOG("Copy vol size %llu, target vol size %llu, %s", copyVolObj.m_volSizeInBytes, sizeInBytes,
            m_taskId.c_str());
        return false;
    }
    DBGLOG("Target volume(%s) check success, %s", targetVolUUID.c_str(), m_taskId.c_str());
    return true;
}

/**
 * @brief 检查卷是否可以执行恢复
 *        1. 检查源卷与目标卷存储架构是否一致(arm/x86)
 *        2. 目标卷状态是否满足恢复要求
 *        3. 目标卷大小必须大于等于副本中卷大小
 * @param copyVolMetaData
 * @return true
 * @return false
 */
bool HCSProtectEngine::CheckTargetVolumeList(const VolInfo &copyVolObj, const ApplicationResource &restoreVol)
{
    Json::Value volExtendInfo;
    // 获取目标卷ID
    if (!Module::JsonHelper::JsonStringToJsonValue(restoreVol.extendInfo, volExtendInfo)) {
        ERRLOG("Convert %s failed, %s", WIPE_SENSITIVE(restoreVol.extendInfo.c_str()), m_taskId.c_str());
        return false;
    }
    std::string targetVolsInfoStr = volExtendInfo["targetVolumes"].asString();
    DBGLOG("TargetVolsInfo %s. %s", targetVolsInfoStr.c_str(), m_taskId.c_str());
    Json::Value targetVolumeList;
    if (!Module::JsonHelper::JsonStringToJsonValue(targetVolsInfoStr, targetVolumeList)) {
        ERRLOG("Transfer %s failed, %s.", targetVolsInfoStr.c_str(), m_taskId.c_str());
        return false;
    }
    for (const auto &item : targetVolumeList) {
        std::string uuid = item["uuid"].asString();
        if (uuid.empty()) {
            ERRLOG("Uuid is empty for vol %s, %s.", restoreVol.id.c_str(), m_taskId.c_str());
            return false;
        }
        if (!CheckTargetVolume(copyVolObj, uuid)) {
            ERRLOG("Target volume check failed %s, %s.", uuid.c_str(), m_taskId.c_str());
            return false;
        }
    }
    DBGLOG("Target volumes check success, %s", m_taskId.c_str());
    return true;
}

bool HCSProtectEngine::CheckVolumeList(const std::vector<VolInfo> &copyVolList)
{
    std::vector<ApplicationResource> volList = m_jobHandle->GetVolumes();
    for (const auto &restoreVol : volList) {
        bool find = false;
        for (const auto &vol : copyVolList) {
            if (vol.m_uuid != restoreVol.id) {
                continue;
            }
            find = true;
            if (!CheckTargetVolumeList(vol, restoreVol)) {
                return false;
            }
            DBGLOG("volume(%s) check success, %s", restoreVol.id.c_str(), m_taskId.c_str());
            break;
        }
        if (!find) {
            ERRLOG("No match volume for restore vol id %s, %s", restoreVol.id, m_taskId.c_str());
            return false;
        }
    }
    return true;
}

int32_t HCSProtectEngine::CheckBeforeBackup()
{
    int32_t ret = SUCCESS;
    TP_START("TP_HCS_CheckBeforeBackupFailed", 1, &ret);
    TP_END
    if (ret == FAILED) {
        return FAILED;
    }
    if (!CheckVMStatus()) {
        return FAILED;
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::GetProjectResource(ResourceResultByPage& page, HcsResourceAccess &access,
    const ListResourceRequest& request, std::string &erroParam)
{
    for (auto app : request.applications) {
        access.SetApplication(app);
        std::string parentId;
        if (access.GetProjectLists(page, parentId, erroParam) == FAILED) {
            ERRLOG("Failed to get project lists. name: %s, taskId: %s", app.name.c_str(), m_taskId.c_str());
            return FAILED;
        }
    }
    return SUCCESS;
}

void HCSProtectEngine::AddInvalidItemInfo(ResourceResultByPage &page, const std::string& extendInfo,
    const std::string &parentId)
{
    ApplicationResource appResouce;
    appResouce.__set_type("InvalidItem");
    appResouce.__set_subType("");
    appResouce.__set_id(Uuid::GenerateUuid());
    appResouce.__set_name("");
    appResouce.__set_parentId(parentId);
    appResouce.__set_parentName("");
    appResouce.__set_extendInfo(extendInfo);
    page.items.push_back(appResouce);
    INFOLOG("Add invaild project success. total size: %d. %s", page.items.size(), m_taskId.c_str());
}

int32_t HCSProtectEngine::GetVirtualMachineResource(ResourceResultByPage& page, HcsResourceAccess &access,
    const ListResourceRequest& request)
{
    for (auto app : request.applications) {
        access.SetApplication(app);
        std::string parentId;
        if (access.GetVirtualMachineList(page, parentId) != SUCCESS) {
            ERRLOG("Failed to get vm list, %s", m_taskId.c_str());
            return FAILED;
        }
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::GetDiskResourceDetail(ResourceResultByPage &page, HcsResourceAccess &access,
    const ListResourceRequest& request)
{
    for (auto app : request.applications) {
        access.SetApplication(app);
        std::string parentId;
        if (access.GetVolumeResourceList(page, parentId) != SUCCESS) {
            ERRLOG("Failed to get vm detail info, %s", m_taskId.c_str());
            return FAILED;
        }
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::GetVDCResource(ResourceResultByPage& page, HcsResourceAccess &access,
    const ListResourceRequest& request)
{
    for (auto app : request.applications) {
        access.SetApplication(app);
        std::vector<std::string> errorUserName;
        std::string parentId;
        if (access.GetVDCResourceList(page, errorUserName, parentId) == FAILED) {
            ERRLOG("Failed to get VDC lists. name: %s, taskId: %s", app.name.c_str(), m_taskId.c_str());
            continue;
        }
        AddVDCErrorMsg(errorUserName, page, parentId);
    }
    return SUCCESS;
}

void HCSProtectEngine::AddVDCErrorMsg(const std::vector<std::string>& errorUserName,
    ResourceResultByPage &page, const std::string &parentId)
{
    if (errorUserName.empty()) {
        return;
    }
    Json::Value jsonErrorName;
    for (const auto& nameStr : errorUserName) {
        jsonErrorName.append(nameStr);
    }
    InvalidVdcInfo invalidVdc;
    invalidVdc.m_errorCode = VDC_QUERY_RESOURCE_FAILED;
    invalidVdc.m_errorMsg = "Failed to get vdc resource.";
    Json::FastWriter jsonWriter;
    invalidVdc.m_invalidUserName = jsonWriter.write(jsonErrorName);
    std::string extendInfo = "";
    if (!Module::JsonHelper::StructToJsonString(invalidVdc, extendInfo)) {
        WARNLOG("Struct to invalid vdc info string failed, %s", m_taskId.c_str());
    }
    AddInvalidItemInfo(page, extendInfo, parentId);
    return;
}

void HCSProtectEngine::CheckApplicationReturnValue(ActionResult &returnValue,
    const ApplicationEnvironment &appEnv, const AppProtect::Application &application)
{
    Json::Reader domainInfoReader;
    Json::Value domainInfoJson;
    if (!domainInfoReader.parse(application.extendInfo, domainInfoJson)) {
        returnValue.__set_code(-1);
        ERRLOG("Failed to read extendInfo, %s", m_taskId.c_str());
        return;
    }
    if (!domainInfoJson.isMember("ip")) {
        DBGLOG("The IP address is not verified, %s", m_taskId.c_str());
        return;
    }
    std::string domainstring = "iam-apigateway-proxy." + appEnv.endpoint;
    char buf[BUF_LIMT];
    struct hostent hostInfo;
    struct hostent *phost = nullptr;
    int hostrrno = 0;
    memset_s(&hostInfo, sizeof(hostInfo), 0, sizeof(hostInfo));
    int ret = gethostbyname_r(domainstring.c_str(), &hostInfo, buf, sizeof(buf), &phost, &hostrrno);
    if (ret != 0 || phost == nullptr) {
        returnValue.__set_code(CHECK_HCS_CONNECT_FAILED);
        returnValue.__set_bodyErr(CHECK_HCS_CONNECT_FAILED);
        ERRLOG("No matching IP address, %s", m_taskId.c_str());
        return;
    }
    std::string ipString = "";
    for (int i = 0; phost->h_addr_list[i]; i++) {
        std::string hostsIp = inet_ntoa(*reinterpret_cast<struct in_addr *>(phost->h_addr_list[i]));
        if (hostsIp.empty()) {
            ERRLOG("The IP address domain name is incorrect, %s", m_taskId.c_str());
            returnValue.__set_code(CHECK_HCS_CONNECT_FAILED);
            returnValue.__set_bodyErr(CHECK_HCS_CONNECT_FAILED);
            return;
        }
        ipString = hostsIp;
    }
    if (ipString == domainInfoJson["ip"].asString()) {
        DBGLOG("Check ip succee. ip : %s, %s", ipString.c_str(), m_taskId.c_str());
    } else {
        returnValue.__set_code(CHECK_HCS_CONNECT_FAILED);
        returnValue.__set_bodyErr(CHECK_HCS_CONNECT_FAILED);
        ERRLOG("Check ip failed. ip: %s, %s", ipString.c_str(), m_taskId.c_str());
    }
    return;
}

int32_t HCSProtectEngine::BackUpAllVolumes(SnapshotInfo &snapshot, std::string &errCode)
{
    if (CheckBeforeCreateSnapshot(m_vmInfo.m_volList) != SUCCESS) {
        ERRLOG("Check before create snapshot failed.");
        return FAILED;
    }
    DBGLOG("Begin to create snapshot. m_volList size: %d. %s", m_vmInfo.m_volList.size(), m_taskId.c_str());
    if (!DoCreateSnapshot(m_vmInfo.m_volList, snapshot, errCode)) {
        ERRLOG("Get machine metadata failed, %s", m_taskId.c_str());
        return FAILED;
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckAttachServerVolume(const VolInfo &volObj)
{
    AttachServerVolumeRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    request.SetServerId(m_jobHandle->GetApp().id);
    request.SetVolumeId(volObj.m_uuid);
    if (!volObj.m_attachPoints.empty()) {
        request.SetDevice(volObj.m_attachPoints[0].m_device);
    } else {
        ERRLOG("Volume(%s) has no attach point. %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    std::shared_ptr<AttachServerVolumeResponse> response = m_ecsClient.AttachServerVolume(request);
    if (response == nullptr) {
        ERRLOG("Send attach server volume request failed, volId: %s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK)) {
        ERRLOG("Get attach server volume response failed, volId: %s, statusCode: %d", volObj.m_uuid.c_str(),
            response->GetStatusCode());
        return FAILED;
    }
    INFOLOG("Send volume(%s) attach msg success, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
    return SUCCESS;
}

int32_t HCSProtectEngine::CheckDetachServerVolume(const VolInfo &volObj)
{
    DetachServerVolumeRequest request;
    if (!FormHttpRequest(request)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    request.SetServerId(m_jobHandle->GetApp().id);
    request.SetVolumeId(volObj.m_uuid);
    std::shared_ptr<DetachServerVolumeResponse> response = m_ecsClient.DetachServerVolume(request);
    if (response == nullptr) {
        ERRLOG("Send detach server volume request failed, volId: %s, %s", volObj.m_uuid.c_str(), m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK) &&
        response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::ACCEPTED)) {
        ERRLOG("Detach volume(%s) failed, status:%d, %s", volObj.m_uuid.c_str(), response->GetStatusCode(),
            m_taskId.c_str());
        return FAILED;
    }
    return SUCCESS;
}

bool HCSProtectEngine::ParseCinderCert(const std::string &cinderCertInfo)
{
    INFOLOG("Start to set cinder cert path.");
    if (m_cinderCertMgr == nullptr) {
        ERRLOG("The cinder CertMgr is nullptr, %s", m_taskId.c_str());
        return false;
    }
    CpsCertInfo cpsCertInfo;
    if (!m_cinderCertMgr->ParseCertInfo(cinderCertInfo, cpsCertInfo)) {
        ERRLOG("Failed to parse cinder cert info.");
        return false;
    }
    if (m_cinderCertMgr->IsVerifyCert()) {
        if (!m_cinderCertMgr->SaveCertToFile(CINDER_CERT_HEAD + m_appEnv.id)) {
            ERRLOG("Save target env cinder cert info to file failed, cinder%s", m_appEnv.id.c_str());
            return false;
        }
    } else {
        m_cinderCertMgr->RemoveCrtFile(CINDER_CERT_HEAD + m_appEnv.id);
        m_cinderCertMgr->RemoveCrlFile(CINDER_CERT_HEAD + m_appEnv.id);
    }
    return true;
}

int32_t HCSProtectEngine::GetSnapshotsOfVolume(const VolInfo &volInfo, std::vector<VolSnapInfo> &snapList)
{
    ShowSnapshotListRequest request;
    if (!FormHttpRequest(request, false)) {
        ERRLOG("Initialize HCS request failed, %s", m_taskId.c_str());
        return FAILED;
    }
    request.SetVolumeId(volInfo.m_uuid);

    std::shared_ptr<ShowSnapshotListResponse> response = m_hcsCinderClient.ShowSnapshotList(request);
    if (response == nullptr) {
        ERRLOG("Show snapshot list response failed, %s", m_taskId.c_str());
        return FAILED;
    }
    SnapshotDetialList snapDetailList = response->GetSnapshotList();
    for (const auto &snapDetail : snapDetailList.m_snapshots) {
        DBGLOG("Snapshot [name]%s[description]%s, %s", snapDetail.m_name.c_str(),
            snapDetail.m_description.c_str(), m_taskId.c_str());
        if (!MatchSnapshotName(snapDetail.m_name) || !MatchSnapshotDescription(snapDetail.m_description)) {
            continue;
        }
        DBGLOG("Found snapshot [name]%s[description]%s needs to be deleted, %s", snapDetail.m_name.c_str(),
            snapDetail.m_description.c_str(), m_taskId.c_str());
        VolSnapInfo volSnap;
        if (!FillUpVolSnapInfo(volInfo, snapDetail, "", volSnap)) {
            continue;
        }
        snapList.push_back(volSnap);
    }
    return SUCCESS;
}

int32_t HCSProtectEngine::GetSnapshotProviderAuth(std::vector<std::string>& proAuth,
    GetSnapshotRequest& request, const VolSnapInfo& volSnap)
{
    request.SetSnapshotId(volSnap.m_snapshotId);
    std::shared_ptr<GetSnapshotResponse> response = m_cinderClient->GetSnapshot(request);
    DBGLOG("Snapshot id %s. task id:%s", volSnap.m_snapshotId.c_str(), m_taskId.c_str());
    if (response == nullptr) {
        ERRLOG("Cinder get snap info failed, %s", m_taskId.c_str());
        return FAILED;
    }
    if (response->GetStatusCode() != static_cast<uint32_t>(HcsExternalStatusCode::OK)) {
        ERRLOG("Show snapshot info failed. status code: %u, task id:%s", response->GetStatusCode(), m_taskId.c_str());
        return FAILED;
    }

    std::string providerInfo = response->GetProviderAuth();
    if (providerInfo.empty()) {
        ERRLOG("Provider auth is empty, %s", m_taskId.c_str());
        return FAILED;
    }
    DBGLOG("Snapshot provider info:%s. task id:%s", providerInfo.c_str(), m_taskId.c_str());
    proAuth.push_back(providerInfo);
    return SUCCESS;
}

bool HCSProtectEngine::IfDeleteLatestSnapShot()
{
    Json::Value extendInfo;
    if (!Module::JsonHelper::JsonStringToJsonValue(m_backupPara->extendInfo, extendInfo)) {
        ERRLOG("Trans json value advanceparams failed, %s", m_taskId.c_str());
        return false;
    }
    if (extendInfo.isMember("keep_snapshot") &&
        extendInfo["keep_snapshot"].asString().compare("false") == SUCCESS) {
        INFOLOG("DeleteLatestSnapShot is true, %s", m_taskId.c_str());
        return true;
    }
    return false;
}

int32_t HCSProtectEngine::CheckBeforeMount()
{
    return SUCCESS;
}

int32_t HCSProtectEngine::CancelLiveMount(const VMInfo &liveVm)
{
    return SUCCESS;
}

int32_t HCSProtectEngine::CreateLiveMount(const VMInfo &copyVm, VMInfo &newVm)
{
    return SUCCESS;
}

int32_t HCSProtectEngine::DealWithVolumeHander(const VolSnapInfo &volSnap)
{
    InitRepoHandler();
    std::string vmInfoFile = m_metaRepoPath + VIRT_PLUGIN_VM_INFO;
    VMInfo vmInfo;
    // 副本删除任务不挂载文件系统，handle为空，vmInfo从任务参数中获取
    if (m_metaRepoHandler == nullptr && m_delCopyPara != nullptr) {
        CopyExtendInfo extendInfo;
        if (!Module::JsonHelper::JsonStringToStruct(m_delCopyPara->extendInfo,
            extendInfo)) {
            ERRLOG("Conv json string to struct failed, %s", m_taskId.c_str());
            return FAILED;
        }
        if (!Module::JsonHelper::JsonStringToStruct(extendInfo.m_hostMetadata,
            vmInfo)) {
            ERRLOG("Conv json string to struct failed, %s", m_taskId.c_str());
            return FAILED;
        }
    } else if (Utils::LoadFileToStructWithRetry(m_metaRepoHandler, vmInfoFile, vmInfo) != SUCCESS) {
        ERRLOG("Load vm info failed.");
        return FAILED;
    }
    VolInfo volumeInfo;
    for (const VolInfo &volInfo : vmInfo.m_volList) {
        if (volInfo.m_uuid != volSnap.m_volUuid) {
            continue;
        }
        volumeInfo = volInfo;
    }
    std::shared_ptr<VolumeHandler> storageVolHandler = nullptr;
    if (GetVolumeHandler(volumeInfo, storageVolHandler) != SUCCESS) {
        ERRLOG("Get volume handle failed.");
        return FAILED;
    }
    if (volSnap.m_datastore.m_type.find(OCEANSTORE_TYPE) != std::string::npos ||
        volSnap.m_datastore.m_type.find(DORADO_TYPE) != std::string::npos) {
        return DeleteSnapshotPreHookInDorado(volSnap, storageVolHandler);
    }
    return DeleteSnapshotPreHookInPacific(volSnap, storageVolHandler);
}

int32_t HCSProtectEngine::DeleteSnapshotPreHook(const VolSnapInfo &volSnap)
{
    m_hcsOpServiceUtils.GetAppEnv(m_appEnv);
    return DealWithVolumeHander(volSnap);
}

bool HCSProtectEngine::CheckIsConsistenSnapshot()
{
    return true;
}

bool HCSProtectEngine::SendDeleteSnapshotMsg(const VolSnapInfo &volSnap)
{
    if (!NormalDeleteSnapshot(volSnap)) {
        ERRLOG("Normal delete snapshot(%s) failed, try forece delete.", volSnap.m_snapshotId.c_str());
        return ForceDeleteSnapshot(volSnap);
    }
    return true;
}

bool HCSProtectEngine::NormalDeleteSnapshot(const VolSnapInfo &volSnap)
{
    DeleteSnapshotPreHook(volSnap);
    OpenStackPlugin::DeleteSnapshotRequest request;
    FormHttpRequest(request);
    request.SetSnapshotId(volSnap.m_snapshotId);
    request.SetApiType(ApiType::EVS);
    request.SetScope(Scope::PROJECT);
    for (int i = 0; i < MAX_EXEC_COUNT; i++) {
        std::shared_ptr<OpenStackPlugin::DeleteSnapshotResponse> response = m_evsClient.DeleteSnapshot(request);
        if (response == nullptr) {
            ERRLOG("Send delete snap(%s) msg failed, retry.", volSnap.m_snapshotId.c_str());
            sleep(COMMON_WAIT_TIME);
            continue;
        }
        if (response->GetStatusCode() == static_cast<uint32_t>(Module::SC_NOT_FOUND)) {
            INFOLOG("Snapshot(%s) deleted.", volSnap.m_snapshotId.c_str());
            return true;
        }
        if (response->GetStatusCode() == static_cast<uint32_t>(Module::SC_ACCEPTED)) {
            INFOLOG("Send delete snap(%s) msg success.", volSnap.m_snapshotId.c_str());
            return true;
        }
        ERRLOG("Send delete snap(%s) msg failed, retry.", volSnap.m_snapshotId.c_str());
        sleep(COMMON_WAIT_TIME);
    }
    return false;
}
}